package com.co.webmagic.task;

import com.co.webmagic.process.NewJobProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * @author ldl.plus
 * @date 2020年07月01日  9:47
 */
// @Component
public class NewDownJob {

    @Autowired
    private NewJobProcessor newJobProcessor;

    @Scheduled(cron = "0/5 * * * * *")
    public void down() {
        Spider.create(newJobProcessor)
                .addUrl("https://search.bilibili.com/all?keyword=%E6%90%9E%E7%AC%91")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(1)
                .run();
    }
}
